{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/dask/config.py:161: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  data = yaml.load(f.read()) or {}\n",
      "/anaconda3/lib/python3.7/site-packages/dask/dataframe/utils.py:13: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n",
      "/anaconda3/lib/python3.7/site-packages/distributed/config.py:20: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  defaults = yaml.load(f)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "#import wfdb as wf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from biosppy.signals import ecg\n",
    "import glob\n",
    "from scipy import signal\n",
    "import pandas as pd\n",
    "from tensorflow.python.keras.layers import Dense, Convolution1D, Convolution2D,MaxPool1D, Flatten, Dropout\n",
    "from tensorflow.python.keras.layers import Input\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers.normalization import BatchNormalization\n",
    "import tensorflow.python.keras\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.python.keras.utils.np_utils import to_categorical\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from collections import Counter\n",
    "from sklearn import preprocessing\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os,glob\n",
    "import math\n",
    "import fnmatch\n",
    "import re\n",
    "import gc\n",
    "import shutil\n",
    "from joblib import Parallel, delayed\n",
    "#import matplotlib.pyplot as plt\n",
    "import scipy.signal as scipysi\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "from sklearn.multiclass import OutputCodeClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.neighbors import NearestCentroid\n",
    "from sklearn.neighbors import RadiusNeighborsClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn import linear_model\n",
    "from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.layers import Bidirectional, GRU\n",
    "import shutil\n",
    "#import matplotlib as plt\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers import Input, Dense, LSTM, multiply, concatenate, Activation, Masking, Reshape,CuDNNLSTM,GlobalMaxPooling1D, MaxPool2D,Flatten\n",
    "from tensorflow.python.keras.layers import Conv1D, Conv2D, BatchNormalization, GlobalAveragePooling1D, Permute, Dropout, GlobalAveragePooling2D,Concatenate\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "from imblearn.over_sampling import SMOTE, ADASYN, SVMSMOTE\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from tensorflow.keras import initializers\n",
    "from tensorflow.keras import regularizers, constraints\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "from scipy import signal\n",
    "from scipy.fft import fftshift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#constants\n",
    "\n",
    "TARGET_SAMPLING_RATE=250\n",
    "DATA_SAMPLING_RATE=500\n",
    "INPUT_BEAT_SIZE=300\n",
    "\n",
    "#paths\n",
    "root_path='/Users/aring/Desktop/ECG-identification/Arnold-data'\n",
    "\n",
    "DataFilePath = root_path+'/NoiseReductionData/'\n",
    "LeadInfoFilePath = root_path+'/DiagnosisFiles/MissingLeadInfo/'\n",
    "RepDiagnosisFileName = root_path+'/DiagnosisFiles/RepatTranslationDiagAfterRemDupAndBDQ_CHS20200531_ToArinG.xlsx'\n",
    "FullDiagnosisFileName = root_path+'/DiagnosisFiles/TranslationDiagAfterRemDupAndBDQ_CHS20200531_ToArinG.xlsx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45308, 7)\n"
     ]
    }
   ],
   "source": [
    "FullDiagnosisDF = pd.read_excel(FullDiagnosisFileName)\n",
    "print(FullDiagnosisDF.shape)\n",
    "AllDiagnosisDF['PatientID'] = AllDiagnosisDF['PatientID'].astype('str')\n",
    "AllDiagnosisDF['PatientID_new'] = le.fit_transform(AllDiagnosisDF['PatientID'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FileName</th>\n",
       "      <th>PatientID</th>\n",
       "      <th>DOB</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gen</th>\n",
       "      <th>TimeAcquisition</th>\n",
       "      <th>Translation</th>\n",
       "      <th>PatientID_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MUSE_20180113_180130_23000</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>85</td>\n",
       "      <td>Male</td>\n",
       "      <td>29/12/2016 10:52:16</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MUSE_20180113_180131_30000</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>29/05/2017 10:10:54</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MUSE_20180113_180139_35000</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>03/06/2017 07:34:27</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MUSE_20180113_180143_59000</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>04/11/2017 11:46:50</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MUSE_20180113_131105_22000</td>\n",
       "      <td>542</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Female</td>\n",
       "      <td>26/01/2017 12:50:33</td>\n",
       "      <td>[ 'Ventricular_Escape_Beat']</td>\n",
       "      <td>2269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     FileName PatientID        DOB Age     Gen  \\\n",
       "0  MUSE_20180113_180130_23000       219  1011931.0  85    Male   \n",
       "1  MUSE_20180113_180131_30000       219  1011931.0  86    Male   \n",
       "2  MUSE_20180113_180139_35000       219  1011931.0  86    Male   \n",
       "3  MUSE_20180113_180143_59000       219  1011931.0  86    Male   \n",
       "4  MUSE_20180113_131105_22000       542  1011931.0  86  Female   \n",
       "\n",
       "       TimeAcquisition                   Translation  PatientID_new  \n",
       "0  29/12/2016 10:52:16         ['Sinus_Tachycardia']           1673  \n",
       "1  29/05/2017 10:10:54         ['Sinus_Tachycardia']           1673  \n",
       "2  03/06/2017 07:34:27         ['Sinus_Tachycardia']           1673  \n",
       "3  04/11/2017 11:46:50         ['Sinus_Tachycardia']           1673  \n",
       "4  26/01/2017 12:50:33  [ 'Ventricular_Escape_Beat']           2269  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AllDiagnosisDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "AllDiagnosisDF['FileName'] = AllDiagnosisDF.apply(lambda x: x['FileName']+'.csv',axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FileName</th>\n",
       "      <th>PatientID</th>\n",
       "      <th>DOB</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gen</th>\n",
       "      <th>TimeAcquisition</th>\n",
       "      <th>Translation</th>\n",
       "      <th>PatientID_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MUSE_20180113_180130_23000.csv</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>85</td>\n",
       "      <td>Male</td>\n",
       "      <td>29/12/2016 10:52:16</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MUSE_20180113_180131_30000.csv</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>29/05/2017 10:10:54</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MUSE_20180113_180139_35000.csv</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>03/06/2017 07:34:27</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MUSE_20180113_180143_59000.csv</td>\n",
       "      <td>219</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Male</td>\n",
       "      <td>04/11/2017 11:46:50</td>\n",
       "      <td>['Sinus_Tachycardia']</td>\n",
       "      <td>1673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MUSE_20180113_131105_22000.csv</td>\n",
       "      <td>542</td>\n",
       "      <td>1011931.0</td>\n",
       "      <td>86</td>\n",
       "      <td>Female</td>\n",
       "      <td>26/01/2017 12:50:33</td>\n",
       "      <td>[ 'Ventricular_Escape_Beat']</td>\n",
       "      <td>2269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         FileName PatientID        DOB Age     Gen  \\\n",
       "0  MUSE_20180113_180130_23000.csv       219  1011931.0  85    Male   \n",
       "1  MUSE_20180113_180131_30000.csv       219  1011931.0  86    Male   \n",
       "2  MUSE_20180113_180139_35000.csv       219  1011931.0  86    Male   \n",
       "3  MUSE_20180113_180143_59000.csv       219  1011931.0  86    Male   \n",
       "4  MUSE_20180113_131105_22000.csv       542  1011931.0  86  Female   \n",
       "\n",
       "       TimeAcquisition                   Translation  PatientID_new  \n",
       "0  29/12/2016 10:52:16         ['Sinus_Tachycardia']           1673  \n",
       "1  29/05/2017 10:10:54         ['Sinus_Tachycardia']           1673  \n",
       "2  03/06/2017 07:34:27         ['Sinus_Tachycardia']           1673  \n",
       "3  04/11/2017 11:46:50         ['Sinus_Tachycardia']           1673  \n",
       "4  26/01/2017 12:50:33  [ 'Ventricular_Escape_Beat']           2269  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AllDiagnosisDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "AllDiagnosisDF[['FileName']].to_csv('/Users/aring/Desktop/ECG-identification/filenames.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45309, 8)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AllDiagnosisDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
